using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Data;
using System.Threading;

namespace Lib.Crawler
{
    public class QuickCrawling
    {
        public QuickCrawling()
        {

        }
        public void Dispose()
        {
            
        }
        /// <summary>
        /// Get the data from link
        /// </summary>
        /// <param name="URL">The web URL</param>
        /// <param name="Path">The Path inside web URL</param>
        /// <returns></returns>
        public ObjCrawler CrawlLink(string URL, string Path)
        {
            // Initiate HTTP object from Utilities
            Utilities.BasicHTTP objHttp = new Lib.Utilities.BasicHTTP();
            string Content = "";
            // Get status from URL, and return response values to Content variable
            HttpStatusCode scode = objHttp.GetLinkStatusAndContent("http://"+URL + Path, ref Content);
            ObjCrawler objCrawl = new ObjCrawler();
            // Check whether HTTP status is ok, if ok, extract data into variables of crawler
            if (scode == HttpStatusCode.OK)
            {
                // Assign HTML content to Content variable
                objCrawl.InnerHTML = Content;
                // Using RE to find the links exist in Content variable (HTML content)
                System.Text.RegularExpressions.MatchCollection match = System.Text.RegularExpressions.Regex.Matches(Content, "(?i)href=[\"'](.+?)[\"']");
                System.Text.RegularExpressions.MatchCollection matchname = System.Text.RegularExpressions.Regex.Matches(Content, "[?>].*[</a>]");
                //>).*(</a)
                // Create an array list
                System.Collections.ArrayList arrList = new System.Collections.ArrayList();
                System.Collections.ArrayList arrListName = new System.Collections.ArrayList();
                // Add Links found inside Content to the array list
                foreach (System.Text.RegularExpressions.Match objMatch in match)
                    arrList.Add(objMatch.Value);
                foreach (System.Text.RegularExpressions.Match objMatch in matchname)
                    arrListName.Add(objMatch.Value);
                // Cast array list to string array and assign to LinksInside properties of ObjCrawler object
                objCrawl.LinksInside = (string[])arrList.ToArray(typeof(string));
                objCrawl.LinksName = (string[])arrListName.ToArray(typeof(string));
            }
            // Return the object ObjCrawler
            return objCrawl;
        }
        int _numRows = 30;
        
        public void CrawlLinks(string URL, string Path)
        {
            objQueu.Enqueue("http://" + URL + Path);
            utbl = new DataTable();
            utbl.Columns.Add("Index", typeof(int));
            utbl.Columns["Index"].AutoIncrement = true;
            utbl.Columns["Index"].AutoIncrementSeed = 1;
            utbl.Columns["Index"].AutoIncrementStep = 1;

            utbl.Columns.Add("Link", typeof(string));
            utbl.Columns.Add("Status", typeof(bool));
            utbl.Columns.Add("Retry", typeof(int));
            utbl.PrimaryKey = new DataColumn[] { utbl.Columns["Link"] };
            //utbl = tbl;
            utbl.Rows.Add(new object[] { null, "http://" + URL + Path, false, 0 });
            rootURL = URL;
            Crawl("");
            _total++;
        }
        string rootURL = "";
        public void Crawl(object obj)
        {
            if (!working)
            {
                _numStop++;
                return;
            }
            string URL = (objQueu.Count > 0) ? (string)objQueu.Dequeue() : null;
            string URLName = ""; // URL;
            if (URL.IndexOf(Environment.NewLine)>=0)
            {
                string[] URLContent = URL.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
                if (URLContent.Length > 0)
                {
                    URL = URLContent[0];
                    if (URLContent.Length > 1)
                        URLName = string.Join(" ", URLContent, 1, URLContent.Length - 1);
                    else
                        URLName = URL;
                }
                else
                    return;
            }

            if (!string.IsNullOrEmpty(URL))
            {
                // Initiate HTTP object from Utilities
                Utilities.BasicHTTP objHttp = new Lib.Utilities.BasicHTTP();

                string Content = "";
                // Get status from URL, and return response values to Content variable
                HttpStatusCode scode = objHttp.GetLinkStatusAndContent(URL, ref Content);
                //ObjCrawler objCrawl = new ObjCrawler();
                // Check whether HTTP status is ok, if ok, extract data into variables of crawler
                if (scode == HttpStatusCode.OK)
                {

                    // Assign HTML content to Content variable
                    //objCrawl.InnerHTML = Content;

                    // Save content to Temporary File
                    Extraction.ToFlatData.ExtractFlatToFile(URL + Environment.NewLine + URLName + Environment.NewLine + Content, "*/*", rootURL);

                    // Using RE to find the links exist in Content variable (HTML content)
                    _success++;
                    lock (utbl)
                        try
                        {
                            utbl.Rows.Find(URL)["Status"] = true;
                        }
                        catch { }
                    string Pattern = @"(?ixn-ms: (<a\s*href\s*=\s*""(?<href>[^""]+)""\s*>)(?<content>[^<]+)(</a>) )";

                    //System.Text.RegularExpressions.MatchCollection match = System.Text.RegularExpressions.Regex.Matches(Content, "(?i)href=[\"'](.+?)[\"']");
                    System.Text.RegularExpressions.MatchCollection match = System.Text.RegularExpressions.Regex.Matches(Content, Pattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase);

                    // Create an array list
                    //System.Collections.ArrayList arrList = new System.Collections.ArrayList();
                    // Add Links found inside Content to the array list
                    foreach (System.Text.RegularExpressions.Match objMatch in match)
                    {
                        string nextURL = objMatch.Groups["href"].ToString();
                        string nameContents = objMatch.Groups["content"].ToString();
                        Utilities.CommonUtility.RemoveStrHtmlTags(nameContents, "");
                        /*
                        string nameContents = Content.Substring(Content.IndexOf(objMatch.Value));
                        if (nameContents.IndexOf("/>") > nameContents.IndexOf('>'))
                            if(nameContents.IndexOf("</a>")<=nameContents.IndexOf("</"))
                                nameContents = nameContents.Substring(nameContents.IndexOf('>')+1, nameContents.IndexOf("</a>") - nameContents.IndexOf('>')-1);

                        if (!string.IsNullOrEmpty(nameContents))
                            Utilities.CommonUtility.RemoveStrHtmlTags(nameContents);
                         */
                        if (nextURL.IndexOf(rootURL.Replace("www.", "")) >= 0)
                        {

                            nextURL = nextURL.Replace("href=", "").Replace("\"", "").Replace("\'", "");
                            if (nextURL.StartsWith("/"))
                                nextURL = rootURL + nextURL.Remove(0, 1);
                            if (nextURL.IndexOf("http://") < 0)
                                nextURL = "http://" + nextURL;
                            if (!utbl.Rows.Contains(nextURL))
                            {
                                utbl.Rows.Add(new object[] { null, nextURL, false, 0 });

                                objQueu.Enqueue(nextURL + Environment.NewLine + nameContents);
                                _total++;
                                //QuickCrawling qc = new QuickCrawling();
                                ThreadPool.QueueUserWorkItem(new WaitCallback(Crawl));
                            }
                        }
                        else
                        {
                            if (nextURL.IndexOf("http://") < 0)
                            {
                                nextURL = nextURL.Replace("href=", "").Replace("\"", "").Replace("\'", "");
                                if (nextURL != URL.Replace(rootURL, "").Replace("http://", "").Replace("www.", ""))
                                {

                                    if (nextURL.StartsWith("/"))
                                        nextURL = rootURL + nextURL;
                                    else
                                        nextURL = rootURL + "/" + nextURL;
                                    if (nextURL.IndexOf("http://") < 0)
                                        nextURL = "http://" + nextURL;
                                    if (!utbl.Rows.Contains(nextURL))
                                    {
                                        try
                                        {
                                            utbl.Rows.Add(new object[] { null, nextURL, false, 0 });
                                        }
                                        catch { }

                                        objQueu.Enqueue(nextURL + Environment.NewLine + nameContents);
                                        _total++;
                                        //QuickCrawling qc = new QuickCrawling();
                                        object state = new object();
                                        ThreadPool.QueueUserWorkItem(new WaitCallback(Crawl), state);
                                    }
                                }
                            }
                        }
                        /*
                        if (objMatch.Value.IndexOf(rootURL.Replace("www.", "")) >= 0)
                        {
                            
                                string nextURL = objMatch.Value.Replace("href=", "").Replace("\"", "").Replace("\'", "");
                                if (nextURL.StartsWith("/"))
                                    nextURL = rootURL + nextURL.Remove(0, 1);
                                if (nextURL.IndexOf("http://") < 0)
                                    nextURL = "http://" + nextURL;
                                if (!utbl.Rows.Contains(nextURL))
                                {
                                    utbl.Rows.Add(new object[] { null, nextURL, false, 0 });

                                    objQueu.Enqueue(nextURL + Environment.NewLine + nameContents);
                                    _total++;
                                    //QuickCrawling qc = new QuickCrawling();
                                    ThreadPool.QueueUserWorkItem(new WaitCallback(Crawl));
                                }
                        }
                        else
                        {
                            if (objMatch.Value.IndexOf("http://") < 0)
                            {
                                string nextURL = objMatch.Value.Replace("href=", "").Replace("\"", "").Replace("\'", "");
                                if (nextURL != URL.Replace(rootURL, "").Replace("http://", "").Replace("www.", ""))
                                {

                                    if (nextURL.StartsWith("/"))
                                        nextURL = rootURL + nextURL;
                                    else
                                        nextURL = rootURL + "/" + nextURL;
                                    if (nextURL.IndexOf("http://") < 0)
                                        nextURL = "http://" + nextURL;
                                    if (!utbl.Rows.Contains(nextURL))
                                    {
                                        utbl.Rows.Add(new object[] { null, nextURL, false, 0 });

                                        objQueu.Enqueue(nextURL + Environment.NewLine + nameContents);
                                        _total++;
                                        //QuickCrawling qc = new QuickCrawling();
                                        object state = new object();
                                        ThreadPool.QueueUserWorkItem(new WaitCallback(Crawl), state);
                                    }
                                }
                            }
                        }
                        */
                        //arrList.Add(objMatch.Value);
                    }
                    // Cast array list to string array and assign to LinksInside properties of ObjCrawler object
                    //objCrawl.LinksInside = (string[])arrList.ToArray(typeof(string));
                }
                else
                {
                    if (scode == HttpStatusCode.BadRequest)
                    {
                        utbl.Rows.Find(URL)["Status"] = false;
                        utbl.Rows.Find(URL)["Retry"] = 3;
                        _failed++;
                    }
                }
            }
        }

        DataTable utbl = new DataTable();

        public DataTable getDataTable()
        {
            return utbl;
        }
        public int _total = 0;
        public int _success = 0;
        public int _failed = 0;
        bool working = true;
        int _numStop = 0;
        public void Pause()
        {
            working = false;
        }
        public void Continue()
        {
            working = true;
            for(int i=0; i<_numStop; i++)
                ThreadPool.QueueUserWorkItem(new WaitCallback(Crawl));
        }

        //System.Collections.Stack objStack = new System.Collections.Stack();
        System.Collections.Queue objQueu = new System.Collections.Queue();
        //System.Collections.ArrayList arrLinkList = new System.Collections.ArrayList();

        private string deQueuLink()
        {
            if (objQueu.Count > 0)
                return (string)objQueu.Dequeue();
            else
                return null;
        }

        //private void pushLink(string linkValue)
        //{
        //    objStack.Push(linkValue);
        //    //arrLinkList.Add(linkValue);
        //}

        //private string popLink()
        //{
        //    if(objStack.Count>0)
        //        return objStack.Pop().ToString();
        //    else
        //        return null;
        //}


    }
}
